package webSpider.MyScrapy;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;

public class Test {
    public static void main(String[] args) throws Exception {
        HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
        //HttpClient  
        CloseableHttpClient closeableHttpClient = httpClientBuilder.build();
        List<String> urls = new ArrayList<String>();
        String urlAddr = "https://movie.douban.com/subject/1292052/";
        HttpGet httpGet = new HttpGet(urlAddr);
        HttpResponse httpResponse = closeableHttpClient.execute(httpGet);
        HttpEntity entity = httpResponse.getEntity();
        InputStream content = entity.getContent();

        HtmlCleaner cleaner = new HtmlCleaner();
        TagNode tagNode = cleaner.clean(content, "utf-8");
        Object[] reUrls = tagNode.evaluateXPath("//div[@class='rating_self clearfix']/strong");

        for (int j = 0; j < reUrls.length; j++) {
            String reUrl = reUrls[j] instanceof TagNode ? ((TagNode) reUrls[j]).getText().toString()
                    : reUrls[j].toString();
            urls.add(reUrl);
        }
        for (String url : urls) {
            System.out.println(url);
        }
    }
}  